package com.xsjiong.learning;

import java.util.Random;
import java.util.*;

public class QLearning {
	private static final boolean LPRINT=false,DPRINT=true;
	private static final int N=6,TARGET=5;
	private static final double RATE=0.8;

	private static Random R=new Random();
	private static int[][] Q=new int[N][N];
	private static int C;

	private static int[][] REWARD=new int[][] {
		{-1,-1,-1,-1,0,-1},
		{-1,-1,-1,0,-1,100},
		{-1,-1,-1,0,-1,-1},
		{-1,0,0,-1,0,-1},
		{0,-1,-1,0,-1,100},
		{-1,0,-1,-1,0,100}
	};

	private static int RandomRoom() {
		int[] tmp=new int[N + 1];
		tmp[0] = 0;
		for (int i=0;i < N;i++) if (REWARD[C][i] >= 0) tmp[++tmp[0]] = i;
		return tmp[0] == 0 ?0: tmp[R.nextInt(tmp[0]) + 1];
	}

	private static int MaxQ(int x) {
		int ret=Integer.MIN_VALUE;
		for (int i=0;i < N;i++) if (Q[x][i] > ret) ret = Q[x][i];
		return ret;
	}

	private static int MaxRoom() {
		int ret=C,max=0;
		for (int i=0;i < N;i++) if (Q[C][i] > max) {
				max = Q[C][i];
				ret = i;
			}
		return ret;
	}

	private static void Learn(int x) {
		C = x;
		int b;
		int c=0;
		do {
			b = RandomRoom();
			Q[C][b] = (int) (REWARD[C][b] + RATE * MaxQ(b));
			c += REWARD[C][b];
			C = b;
			if (LPRINT) System.out.print(" -> " + C);
		} while (C != TARGET);
		if (LPRINT) System.out.print(" R:" + c);
	}

	private static void Decide(int x) {
		C = x;
		int b;
		do {
			b = MaxRoom();
			/*if (b == C) C = RandomRoom(); else
			*/	C = b;
			if (DPRINT) System.out.print(" -> " + C);
		} while (C != TARGET);
	}

	private static void PrintQ() {
		for (int i=0,j;i < N;i++) {
			for (j = 0;j < N;j++)
				System.out.print(Q[i][j] + " ");
			System.out.println();
		}
	}

	public static void main(String[] args) {
		int a;
		for (int i=1;i <= 1000;i++) {
			a = R.nextInt(N);
			if (LPRINT) System.out.print(a);
			Learn(a);
			if (LPRINT) System.out.println();
		}
		PrintQ();
		Scanner S=new Scanner(System.in);
		int n;
		while (true) {
			n = S.nextInt();
			if (n < 0 || n >= N) continue;
			if (DPRINT) System.out.print(n);
			Decide(n);
			if (DPRINT) System.out.println();
		}
	}
}
